package qf.index;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ConcurrentHashSet;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSONObject;

import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

public class ReadFenLei {
    /**
     * 将数据分为四种类型：中国医院、临床指南、PubMed、自2019年以来四种，在同一张表上，用于不同的四款产品。并产生四款产品均需要的STEM或复旦排行榜学科下的，相关机构、相关期刊、相关关键词、相关作者四种默认数据。
     * 贾新志 2025.02.08
     */
    public static void main(String[] args) throws SQLException {
        String newDataName = "scholar_25_02.";
        String summaryTable = newDataName + "unified_search_summary_0120";
        String cnDeptSumTable = newDataName + "cn_dept_sum_dy30";
        String cnFullSumTable = newDataName + "cn_full_sum_dy300";
        String pmArticleTable = newDataName + "spider_pubmed_vip_extend";
        String summaryTypeTable = newDataName + "unified_search_summary_0120_type";

        int onePageNum = 1000;
        String limit = "";
//        String limit = " limit 1";

        String filePath = "F:\\WorkSpaces\\Data\\src\\main\\resources\\dict\\2015年及以来PubMed收录的临床指南_按Guideline限定_共13598条_2025_01_12.txt";
        updateFourItems(summaryTable, onePageNum, limit, "id", "year`,`pmId`,`doi`,`wos_id`,`c1_au`,`rp_au", true, 10, pmArticleTable, filePath);
        insertSummaryTypeTable(summaryTable, onePageNum, limit, "id", "is_6year`,`c1_au`,`rp_au`,`is_china_hosp`,`is_guidelines`,`is_pubmed_article`,`year", true, 10, summaryTypeTable);

    }

    public static void addFenLei(String summaryTable, int onePageNum, String limit, String pmArticleTable, String filePath, String summaryTypeTable) throws SQLException {
        updateFourItems(summaryTable, onePageNum, limit, "id", "year`,`pmId`,`doi`,`wos_id`,`c1_au`,`rp_au", true, 10, pmArticleTable, filePath);
        insertSummaryTypeTable(summaryTable, onePageNum, limit, "id", "is_6year`,`c1_au`,`rp_au`,`is_china_hosp`,`is_guidelines`,`is_pubmed_article`,`year", true, 10, summaryTypeTable);
    }

    public static void insertSummaryTypeTable(String summaryTable, int onePageNum, String limit, String autoId, String field, boolean print, int allTotal, String summaryTypeTable) throws SQLException {
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        List<Entity> tableNum = Db.use().query("select count(1) as count from " + summaryTable);
        if (tableNum.get(0).getInt("count") < 1) {
            return;
        }
        Set<String> stemSet = uSet.getStemSet();
        Set<String> fuDanSet = uSet.getFuDanSet();
        Set<String> hospSet = uSet.getHospSet();
        Set<String> removeSet = uSet.getRemoveSet();
        Set<String> nameSet = cList.getProvName();
        Set<String[]> removeThirdSet = removeThirdList();
        Map<String, Set<Integer>> typeAndIdSetMap = new ConcurrentHashMap<>();
        int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + summaryTable).intValue();
        Collection<Integer> tableList = new ArrayList<>();
        NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
        AtomicInteger sumNum = new AtomicInteger(0);
        int showNum = Math.max((tableList.size() / allTotal), 1) + (tableList.size() % allTotal);
        AtomicInteger startNum = new AtomicInteger(1);
        tableList.parallelStream().forEach(startId -> {
            try {
                List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + summaryTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                if (print && sumNum.getAndIncrement() % showNum == 0) {
                    System.out.println("\033[31;1m" + format.format(new Date()) + " ⒀ 插入分类表，现在开始：" + startNum.getAndIncrement() + " 总次数：" + allTotal + " 表名：" + summaryTable + Utils.printMemoryStr() + "\033[0m");
                }
                for (Entity entry : tableData) {
                    Integer id = entry.getInt(autoId);
                    Integer is6Year = entry.getInt("is_6year");
                    if (null == is6Year || is6Year < 1 || null == id || id < 1) {
                        continue;
                    }
                    String c1Au = Utils.getStrField(entry, "c1_au");
                    String rpAu = Utils.getStrField(entry, "rp_au");
                    List<En.C1Auth> c1List = (c1Au.contains("sort")) ? JSONObject.parseArray(c1Au, En.C1Auth.class) : new ArrayList<>();
                    List<En.C1Auth> rpList = (rpAu.contains("sort")) ? JSONObject.parseArray(rpAu, En.C1Auth.class) : new ArrayList<>();
                    // 类型分4种，1:中国医院，2:指南，3:pubmed，4:自2019年以来
                    if (Utils.getIntField(entry, "year") >= 2024) {
                        putTypeAndIdSetMap(id, c1List, "1", entry.getInt("is_china_hosp"), typeAndIdSetMap, stemSet, fuDanSet);
                        putTypeAndIdSetMap(id, c1List, "3", entry.getInt("is_pubmed_article"), typeAndIdSetMap, stemSet, fuDanSet);
                        putTypeAndIdSetMap(id, c1List, "4", is6Year, typeAndIdSetMap, stemSet, fuDanSet);
                        putTypeAndIdSetMap(id, rpList, "1", entry.getInt("is_china_hosp"), typeAndIdSetMap, stemSet, fuDanSet);
                        putTypeAndIdSetMap(id, rpList, "3", entry.getInt("is_pubmed_article"), typeAndIdSetMap, stemSet, fuDanSet);
                        putTypeAndIdSetMap(id, rpList, "4", is6Year, typeAndIdSetMap, stemSet, fuDanSet);
                    }
                    putTypeAndIdSetMap(id, c1List, "2", entry.getInt("is_guidelines"), typeAndIdSetMap, stemSet, fuDanSet);
                    putTypeAndIdSetMap(id, rpList, "2", entry.getInt("is_guidelines"), typeAndIdSetMap, stemSet, fuDanSet);
                }
            } catch (SQLException e) {
                e.printStackTrace();
            }
        });
        Db.use().execute("truncate table " + summaryTypeTable);
        AtomicInteger number = new AtomicInteger(1);
        typeAndIdSetMap.keySet().parallelStream().forEach(s -> {
            Set<Integer> idSet = typeAndIdSetMap.get(s);
//            System.out.println("\033[31;1m" + format.format(new Date()) + " 插入数据，现在处理：" + number.getAndIncrement() + " 科目为: " + s + " 总条数：" + typeAndIdSetMap.size() + " 本次篇数：" + idSet.size() + Utils.printMemoryStr() + "\033[0m");
            try {
                String[] split = s.split("Łł");
                if (split.length == 3) {
                    int type = NumberUtil.isInteger(split[0]) ? Integer.parseInt(split[0]) : 0;
                    int stemFuDan = NumberUtil.isInteger(split[1]) ? Integer.parseInt(split[1]) : 0;
                    // stemFuDan为1时，表明后面的字符串为STEM专科，为2时，表明后面的字符串为复旦排行榜学科。因为，我们只能要该专科时的作者或机构（而不是该篇论文中有这个专科，所有的作者和机构都计算到该专科下），关键词和期刊则不受影响。如果是中国医院，还要求所有的c1也均是中国的医院才计算作者和机构，否则不计算。
                    En.FourItemsList fourItemsList = inSearch(summaryTable, idSet, stemFuDan, split[2], type == 1, nameSet, hospSet, removeSet, removeThirdSet);
                    List<En.FourItems> keyWordList = fourItemsList.getKeyWordList();
                    List<En.FourItems> unitList = fourItemsList.getUnitList();
                    List<En.FourItems> authList = fourItemsList.getAuthList();
                    List<En.FourItems> journalList = fourItemsList.getJidList();
                    try {
                        Db.use().insert(Entity.create(summaryTypeTable).set("type", type).set("stem_fudan", stemFuDan).set("dept", split[2]).set("id_list", JSONUtil.toJsonStr(idSet)).set("id_sum", idSet.size())
                                .set("key_word", JSONUtil.toJsonStr(keyWordList)).set("auth", JSONUtil.toJsonStr(authList))
                                .set("unit", JSONUtil.toJsonStr(unitList)).set("journal", JSONUtil.toJsonStr(journalList))
                                .set("key_word_20", JSONUtil.toJsonStr(keyWordList.size() > 20 ? new ArrayList<>(keyWordList.subList(0, 20)) : keyWordList))
                                .set("auth_20", JSONUtil.toJsonStr(authList.size() > 20 ? new ArrayList<>(authList.subList(0, 20)) : authList))
                                .set("unit_20", JSONUtil.toJsonStr(unitList.size() > 20 ? new ArrayList<>(unitList.subList(0, 20)) : unitList))
                                .set("journal_20", JSONUtil.toJsonStr(journalList.size() > 20 ? new ArrayList<>(journalList.subList(0, 20)) : journalList)));
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }
                }
            } catch (SQLException e) {
//                throw new RuntimeException(e);
            }
        });
        stemSet.clear();
        fuDanSet.clear();
        hospSet.clear();
        removeSet.clear();
        nameSet.clear();
        removeThirdSet.clear();
        typeAndIdSetMap.clear();
        tableList.clear();
    }

    public static void putTypeAndIdSetMap(int id, List<En.C1Auth> c1OrRpList, String type, Integer number, Map<String, Set<Integer>> typeAndIdSetMap, Set<String> stemSet, Set<String> fuDanSet) {
        // 分三种种，1:STEM专科，2:复旦排行榜学科，3:教育部学科，大类在前，小类在后，中间用"_"分隔。
        if (null != number && number > 0 && !c1OrRpList.isEmpty()) {
            for (En.C1Auth c1Auth : c1OrRpList) {
                Set<String> stems = c1Auth.getStemSet();
                for (String stem : stems) {
                    if (stemSet.contains(stem)) {
                        Utils.putStrSetInt(type + "Łł1Łł" + stem, id, typeAndIdSetMap);
                    }
                }
                Set<String> fuDans = c1Auth.getFuDanSet();
                for (String fuDan : fuDans) {
                    if (fuDanSet.contains(fuDan)) {
                        Utils.putStrSetInt(type + "Łł2Łł" + fuDan, id, typeAndIdSetMap);
                    }
                }
                Set<En.Edu> eduSet = c1Auth.getEduSet();
                for (En.Edu edu : eduSet) {
                    String category = edu.getCategory();
                    String subject = edu.getSubject();
                    if (category.length() > 1 && subject.length() > 1) {
                        Utils.putStrSetInt(type + "Łł3Łł" + category + "_" + subject, id, typeAndIdSetMap);
                    }
                }
            }
        }
    }

    public static void updateFourItems(String summaryTable, int onePageNum, String limit, String autoId, String field, boolean print, int allTotal, String pmArticleTable, String filePath) throws SQLException {
        // 更新四项值，分别是is_china_hosp、is_guidelines、is_pubmed、is_6year。中国医院部分只要WOS的数据，临床指南只要pubmed收录的临床指南，pubmed部分采用原pubmed表中的doi及本表的pmId > 0，没验证是否有漏的，is_6year则是自2019年开始的所有数据。
        List<Entity> tableNum = Db.use().query("select count(1) as count from " + summaryTable);
        if (tableNum.get(0).getInt("count") < 1) {
            return;
        }
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Set<String> hospSet = uSet.getHospSet();
        Set<String> removeSet = uSet.getRemoveSet();
        Set<String> nameSet = cList.getProvName();
        Set<String> pmDoiSet = getPmDoiSet(pmArticleTable, onePageNum, limit, "id", "doi");
        Set<Integer> guidelinesPmIdSet = new HashSet<>();
        Set<String> guidelinesDoiSet = new HashSet<>();
        List<String> list = FileUtil.readLines(filePath, "utf-8");
        for (String s : list) {
            String[] split = s.split("\\|");
            String pm = split.length == 2 ? split[0] : "";
            String doi = split.length == 2 ? split[1] : "";
            pm = split.length == 1 ? split[0] : pm;
            if (NumberUtil.isInteger(pm)) {
                guidelinesPmIdSet.add(Integer.parseInt(pm));
            }
            if (doi.startsWith("10.")) {
                guidelinesDoiSet.add(doi.toLowerCase().trim());
            }
        }
        int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + summaryTable).intValue();
        Collection<Integer> tableList = new ArrayList<>();
        NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
        AtomicInteger sumNum = new AtomicInteger(0);
        int showNum = Math.max((tableList.size() / allTotal), 1) + (tableList.size() % allTotal);
        AtomicInteger startNum = new AtomicInteger(1);
        tableList.parallelStream().forEach(startId -> {
            try {
                List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + summaryTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                if (print && sumNum.getAndIncrement() % showNum == 0) {
                    System.out.println("\033[31;1m" + format.format(new Date()) + " ⑿ 更新四项值，现在开始：" + startNum.getAndIncrement() + " 总次数：" + allTotal + " 表名：" + summaryTable + Utils.printMemoryStr() + "\033[0m");
                }
                for (Entity entry : tableData) {
                    Integer year = entry.getInt("year");
                    Integer id = entry.getInt(autoId);
                    if (null == year || year < 2019 || null == id || id < 1) {
                        continue;
                    }
                    int pmId = Utils.getIntField(entry, "pmId");
                    String doi = Utils.dellEnd(Utils.getStrField(entry, "doi")).toLowerCase();
                    int wosId = Utils.getIntField(entry, "wos_id");
                    String c1Au = Utils.getStrField(entry, "c1_au");
                    String rpAu = Utils.getStrField(entry, "rp_au");
                    List<En.C1Auth> c1List = (c1Au.contains("sort")) ? JSONObject.parseArray(c1Au, En.C1Auth.class) : new ArrayList<>();
                    List<En.C1Auth> rpChangeC1 = (rpAu.contains("sort")) ? JSONObject.parseArray(rpAu, En.C1Auth.class) : new ArrayList<>();
                    int isChinaHosp = wosId > 0 && (ChinaHosp.isChinaHosp(c1List, nameSet, hospSet, removeSet) || ChinaHosp.isChinaHosp(rpChangeC1, nameSet, hospSet, removeSet)) ? 1 : 0;
                    int isGuidelines = (guidelinesPmIdSet.contains(pmId) || guidelinesDoiSet.contains(doi)) ? 1 : 0;
                    int isPubMed = (pmId > 0 || pmDoiSet.contains(doi)) ? 1 : 0;
                    try {
                        Db.use().update(Entity.create().set("is_china_hosp", isChinaHosp).set("is_guidelines", isGuidelines).set("is_pubmed_article", isPubMed).set("is_6year", 1), Entity.create(summaryTable).set(autoId, id));
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }
                }
            } catch (SQLException e) {
                e.printStackTrace();
            }
        });
        hospSet.clear();
        removeSet.clear();
        nameSet.clear();
        pmDoiSet.clear();
        guidelinesPmIdSet.clear();
        guidelinesDoiSet.clear();
        list.clear();
        tableList.clear();
    }

    public static void inSearch1k(String summaryTable, List<Integer> idList, int stemFuDan, String dept, boolean isChinaHosp, Set<String> nameSet, Set<String> hospSet, Set<String> removeSet, Set<String[]> removeThirdSet, En.FourItemsMap f) throws SQLException {
        List<Entity> tableData = Db.use().query("select `zky_dalei`,`aid`,`jid`,`ti_key`,`kw_key`,`abb_word`,`ab_key`,`c1_au`,`rp_au` from " + summaryTable + " where id in(" + CollUtil.join(idList, ",") + ");");
        for (Entity entry : tableData) {
            int zkyDaLei = Utils.getIntField(entry, "zky_dalei");
            int aid = Utils.getIntField(entry, "aid");
            String jid = Integer.toString(Utils.getIntField(entry, "jid"));
            double fenQuXiShu = 0.1;
            fenQuXiShu = zkyDaLei == 1 ? 1.0 : fenQuXiShu;
            fenQuXiShu = zkyDaLei == 2 ? 0.5 : fenQuXiShu;
            fenQuXiShu = zkyDaLei == 3 ? 0.33 : fenQuXiShu;
            fenQuXiShu = zkyDaLei == 4 ? 0.25 : fenQuXiShu;
            addKeyWord(entry.getStr("ti_key"), fenQuXiShu, 0.3, aid, f.getKeyWordAndScoreMap(), f.getKeyWordAndAidSetMap());
            addKeyWord(entry.getStr("kw_key"), fenQuXiShu, 0.3, aid, f.getKeyWordAndScoreMap(), f.getKeyWordAndAidSetMap());
            addKeyWord(entry.getStr("abb_word"), fenQuXiShu, 0.39, aid, f.getKeyWordAndScoreMap(), f.getKeyWordAndAidSetMap());
            addKeyWord(entry.getStr("ab_key"), fenQuXiShu, 0.01, aid, f.getKeyWordAndScoreMap(), f.getKeyWordAndAidSetMap());
            addAuthAndUnit(entry.getStr("c1_au"), fenQuXiShu, "c1", aid, f.getAuthAndAidSetMap(), f.getAuthAndScoreMap(), f.getUnitListAndScoreMap(),
                    f.getUnitListAndAidSetMap(), stemFuDan, dept, isChinaHosp, nameSet, hospSet, removeSet, removeThirdSet);
            addAuthAndUnit(entry.getStr("rp_au"), fenQuXiShu, "rp", aid, f.getAuthAndAidSetMap(), f.getAuthAndScoreMap(), f.getUnitListAndScoreMap(),
                    f.getUnitListAndAidSetMap(), stemFuDan, dept, isChinaHosp, nameSet, hospSet, removeSet, removeThirdSet);
            double finalFenQuXiShu = fenQuXiShu;
            f.getJidAndScoreMap().compute(jid, (key, value) -> value != null ? (value + finalFenQuXiShu) : finalFenQuXiShu);
            Utils.putStrSetInt(jid, aid, f.getJidAndAidSetMap());
        }
    }

    public static En.FourItemsList inSearch(String summaryTable, Set<Integer> idSet, int stemFuDan, String dept, boolean isChinaHosp, Set<String> nameSet, Set<String> hospSet, Set<String> removeSet, Set<String[]> removeThirdSet) throws SQLException {
        // 执行in查询，获取四项（机构、期刊、关键词、作者）的得分及aid的集合，注意，这里in查询的是自增主键，而保存到四项的是aid，不一样。
        En.FourItemsMap fourItemsMap = new En.FourItemsMap();
        List<Integer> idList = new ArrayList<>();
        for (Integer id : idSet) {
            idList.add(id);
            if (idList.size() == 1000) {
                inSearch1k(summaryTable, idList, stemFuDan, dept, isChinaHosp, nameSet, hospSet, removeSet, removeThirdSet, fourItemsMap);
                idList.clear();
            }
        }
        if (!idList.isEmpty()) {
            inSearch1k(summaryTable, idList, stemFuDan, dept, isChinaHosp, nameSet, hospSet, removeSet, removeThirdSet, fourItemsMap);
        }
        // 将四项内容转换成实体类，关键词、期刊、机构、作者，下面每个又包含：名称、aid集合、得分、排序。
        En.FourItemsList fourItemsList = new En.FourItemsList();
        setFourItems(fourItemsMap.getKeyWordAndAidSetMap(), fourItemsMap.getKeyWordAndScoreMap(), fourItemsList.getKeyWordList());
        setFourItems(fourItemsMap.getUnitListAndAidSetMap(), fourItemsMap.getUnitListAndScoreMap(), fourItemsList.getUnitList());
        setFourItems(fourItemsMap.getAuthAndAidSetMap(), fourItemsMap.getAuthAndScoreMap(), fourItemsList.getAuthList());
        setFourItems(fourItemsMap.getJidAndAidSetMap(), fourItemsMap.getJidAndScoreMap(), fourItemsList.getJidList());
        return fourItemsList;
    }

    public static void setFourItems(Map<String, Set<Integer>> keyWordAndAidSetMap, Map<String, Double> keyWordAndScoreMap, List<En.FourItems> keyWordList) {
        Map<String, Double> mapSortValue = Utils.mapSortValue(keyWordAndScoreMap);
        for (Map.Entry<String, Double> entry : mapSortValue.entrySet()) {
            En.FourItems fourItems = new En.FourItems();
            fourItems.setName(Utils.toTitleCase(entry.getKey()));
            fourItems.setAidList(new ArrayList<>(keyWordAndAidSetMap.getOrDefault(entry.getKey(), new HashSet<>())));
            fourItems.setScore(Utils.takeFourDigits(entry.getValue()));
            keyWordList.add(fourItems);
        }
    }

    public static void addAuthAndUnit(String c1OrRp, double fenQuXiShu, String type, int aid, Map<String, Set<Integer>> authAndAidSetMap, Map<String, Double> authAndScoreMap, Map<String, Double> unitListAndScoreMap, Map<String, Set<Integer>> unitListAndAidSetMap,
                                      int stemFuDan, String dept, boolean isChinaHosp, Set<String> nameSet, Set<String> hospSet, Set<String> removeSet, Set<String[]> removeThirdSet) {
        if (null != c1OrRp && c1OrRp.contains("sort")) {
            List<En.C1Auth> c1OrRpList = JSONObject.parseArray(c1OrRp, En.C1Auth.class);
            for (En.C1Auth c1Auth : c1OrRpList) {
                // 如果是中国医院，且判断该c1是中国医院，则继续，如果不是，则跳出不作处理。为什么要这样呢？原因是该篇论文可能有多个作者单位，我们只要中国医院的，其它的不要。
                if (isChinaHosp) {
                    if (!ChinaHosp.c1IsChinaHosp(c1Auth.getInfo().toLowerCase().trim(), nameSet, hospSet, removeSet)) {
                        continue;
                    }
                }
                Set<String> stemSet = c1Auth.getStemSet();
                Set<String> fuDanSet = c1Auth.getFuDanSet();
                if ((stemFuDan == 1 && stemSet.contains(dept)) || (stemFuDan == 2 && fuDanSet.contains(dept)) || (stemFuDan == 3 && dept.contains("_"))) {
                    int sort = c1Auth.getSort();
                    // rp的第一作者，0.3，参与作者0.1。c1的第一作者，0.4，参与作者0.2。为什么c1的得分大于rp？原因是部分数据没有rp，如果不大于，将导致有rp的得分严重高。
                    double score = "c1".equals(type) ? 0.2 : 0.1;
                    score = "c1".equals(type) && sort == 1 ? 0.4 : score;
                    score = "rp".equals(type) && sort == 1 ? 0.3 : score;
                    double finalScore = score;
                    List<String> authList = c1Auth.getAuthList();
                    Set<String> unitSet = c1Auth.getUnitSet();
                    if (null != unitSet && !unitSet.isEmpty()) {
                        // 将机构转换成先大学再医院的字符串格式
                        String unit = CheckUnit.listToUnit(new ArrayList<>(unitSet));
                        unitListAndScoreMap.compute(unit, (key, value) -> value != null ? (value + finalScore * fenQuXiShu) : finalScore * fenQuXiShu);
                        Utils.putStrSetInt(unit, aid, unitListAndAidSetMap);
                    }
                    for (String auth : authList) {
                        auth = auth.toLowerCase().trim();
                        authAndScoreMap.compute(auth, (key, value) -> value != null ? (value + finalScore * fenQuXiShu) : finalScore * fenQuXiShu);
                        Utils.putStrSetInt(auth, aid, authAndAidSetMap);
                    }
                }
            }
        }
    }

    public static List<String> dellRepeatList(List<String> originalList, Set<String[]> removeThirdSet) {
        // 如果list内存在某元素包含另一元素，则将被包含的给予删除。[Kaohsiung Medical University, Kaohsiung Medical University Hospital] -> [Kaohsiung Medical University Hospital]
        List<String> list = new ArrayList<>();
        for (String s : originalList) {
            list.add(s.toLowerCase().trim());
        }
        removeThird(removeThirdSet, list);
        List<String> result = new ArrayList<>(list);
        for (String s : list) {
            for (String t : list) {
                if (!s.equals(t) && s.contains(t)) {
                    result.remove(t);
                }
            }
        }
        Collections.sort(result);
        return result;
    }

    public static void removeThird(Set<String[]> removeThirdSet, List<String> list) {
        // 移除list中的第三个元素。removeThirdSet内每一条均为三个元素，且第三个元素就是要删除的，如果输入的list全部包含了这三个元素，则将第三个元素给予移除，只保留前两个。
        if (list.size() == 3) {
            for (String[] split : removeThirdSet) {
                if (split.length == 3) {
                    if (list.contains(split[0]) && list.contains(split[1]) && list.contains(split[2])) {
                        list.remove(split[2]);
                        break;
                    }
                }
            }
        }
    }

    public static Set<String[]> removeThirdList() {
        String t = "henan cancer hospital|zhengzhou university|affiliated cancer hospital\n" +
                "central south university|hunan cancer hospital|affiliated cancer hospital\n" +
                "jiangsu cancer hospital|nanjing medical university|affiliated cancer hospital\n" +
                "nanjing drum tower hospital|nanjing university|affiliated hospital\n" +
                "jiangsu province hospital chinese medical|nanjing university chinese medical|affiliated hospital\n" +
                "hubei university arts & science|xiangyang central hospital|affiliated hospital\n" +
                "shenzhen peoples hospital 2|shenzhen university|affiliated hospital 1\n" +
                "hunan normal university|hunan province peoples hospital|affiliated hospital 1\n" +
                "jiangsu province hospital|nanjing medical university|affiliated hospital 1\n" +
                "hangzhou medical college|zhejiang province peoples hospital|affiliated peoples hospital\n" +
                "nanjing medical university|suzhou municipal hospital|affiliated suzhou hospital\n" +
                "china medical university|liaoning cancer hospital & institute|cancer hospital\n" +
                "university chinese academy science|zhejiang cancer hospital|cancer hospital\n" +
                "affiliated hospital 1|sun yat sen university|guangdong\n" +
                "affiliated hospital 3|sun yat sen university|guangdong\n" +
                "sun yat sen memorial hospital|sun yat sen university|guangdong\n" +
                "sun yat sen university|zhongshan ophthalm center|guangdong\n" +
                "affiliated hospital 6|sun yat sen university|guangdong\n" +
                "affiliated hospital 7|sun yat sen university|guangdong\n" +
                "affiliated hospital 5|sun yat sen university|guangdong\n" +
                "hospital stomatol|sun yat sen university|guangdong\n" +
                "affiliated hospital 8|sun yat sen university|guangdong\n" +
                "jinan university|shenzhen peoples hospital|guangdong\n" +
                "affiliated hospital 2|sun yat sen university|guangdong\n" +
                "hainan general hospital|hainan medical university|hainan affiliated hospital\n" +
                "sichuan university|west china hospital|mental health center\n" +
                "childrens hospital|fudan university|nacional childrens medical center\n" +
                "john radcliffe hospital|university oxford|oxford ox3 9du\n" +
                "hangzhou medical college|zhejiang province peoples hospital|peoples hospital\n" +
                "henan province peoples hospital|zhengzhou university|peoples hospital\n" +
                "fudan university|huashan hospital|shanghai medical college\n" +
                "china medical university|shengjing hospital|shenyang\n" +
                "army medical university|southeast hospital|third military medical university\n" +
                "huazhong university science & technol|tongji hospital|tongji medical college\n" +
                "huazhong university science & technol|union hospital|tongji medical college\n" +
                "central hospital wuhan|huazhong university science & technol|tongji medical college\n" +
                "huazhong university of science and technology|tongji hospital|tongji medical college\n" +
                "huazhong university of science and technology|union hospital|tongji medical college\n" +
                "toronto general hospital|university toronto|university health network\n" +
                "toronto western hospital|university toronto|university health network\n" +
                "princess margaret cancer center|university toronto|university health network\n" +
                "首都医科大学|首都医科大学附属北京儿童医院|国家儿童医学中心\n" +
                "上海交通大学|上海交通大学医学院附属第九人民医院|口腔医学院\n" +
                " // 以上是出现1000次以上的，以下是出现1000次以下100次以上的。\n" +
                "beth israel deaconess medical center|harvard medical school|[@[ĺbidmc.harvard.eduļ]]\n" +
                "central south university|second xiangya hospital|[@[ĺcsu.edu.cnļ]]\n" +
                "central south university|xiangya hospital|[@[ĺcsu.edu.cnļ]]\n" +
                "beijing|tsinghua university|[@[ĺmails.tsinghua.edu.cnļ]]\n" +
                "feinberg school of medicine|northwestern university|[@[ĺnorthwestern.eduļ]]\n" +
                "beijing|tsinghua university|[@[ĺtsinghua.edu.cnļ]]\n" +
                "beth israel deaconess medical center|harvard medical school|[@electronic address: [ĺbidmc.harvard.eduļ]]\n" +
                "brigham and women's hospital|harvard medical school|[@electronic address: [ĺbwh.harvard.eduļ]]\n" +
                "harvard medical school|massachusetts general hospital|[@electronic address: [ĺmgh.harvard.eduļ]]\n" +
                "rosen college hospitality management|university central florida|9907 university boulevard\n" +
                "guangzhou huiai hospital|guangzhou medical university|affiliated brain hospital\n" +
                "sichuan cancer hospital & institute|university electrical science & technol china|affiliated cancer hospital\n" +
                "huzhou central hospital|huzhou university|affiliated central hospital\n" +
                "xi an jiao tong university|xian childrens hospital|affiliated childrens hospital\n" +
                "nantong university|peoples hospital danyang|affiliated danyang hospital\n" +
                "soochow university|suzhou guangji hospital|affiliated guangji hospital\n" +
                "kunming university science & technol|peoples hospital yunnan province 1|affiliated hospital\n" +
                "peoples hospital chengdu 3|southeast jiaotong university|affiliated hospital\n" +
                "fujian matern & child health hospital|fujian medical university|affiliated hospital\n" +
                "huangshi central hospital|hubei polytech university|affiliated hospital\n" +
                "jinling hospital|nanjing university|affiliated hospital\n" +
                "hospital jiaxing 1|jiaxing university|affiliated hospital\n" +
                "fujian medical university|fujian province matern & childrens hospital|affiliated hospital\n" +
                "nanjing stomatol hospital|nanjing university|affiliated hospital\n" +
                "northwestern university|xian hospital 3|affiliated hospital\n" +
                "nanjing drum tower hospital|nanjing university medical school|affiliated hospital\n" +
                "qingdao university|yantai yuhuangding hospital|affiliated hospital\n" +
                "nanjing jinling hospital|nanjing university|affiliated hospital\n" +
                "hospital wuhan 6|jianghan university|affiliated hospital\n" +
                "northwestern university|xian 3 hospital|affiliated hospital\n" +
                "jiangsu province hospital tradit chinese medical|nanjing university chinese medical|affiliated hospital\n" +
                "nanjing university|suzhou hospital|affiliated hospital\n" +
                "wannan medical college|yijishan hospital|affiliated hospital 1\n" +
                "jiangxi province peoples hospital|nanchang medical college|affiliated hospital 1\n" +
                "zhejiang chinese medical university|zhejiang province hospital chinese medical|affiliated hospital 1\n" +
                "zhejiang chinese medical university|zhejiang province hospital tradit chinese medical|affiliated hospital 1\n" +
                "shenzhen peoples hospital|southern university science & technol|affiliated hospital 1\n" +
                "shandong first medical university|shandong province qianfoshan hospital|affiliated hospital 1\n" +
                "anhui province hospital|university science & technol china|affiliated hospital 1\n" +
                "hubei university science & technol|xianning central hospital|affiliated hospital 1\n" +
                "xiamen cancer hospital|xiamen university|affiliated hospital 1\n" +
                "peoples hospital shenzhen 2|shenzhen university|affiliated hospital 1\n" +
                "shengzhou peoples hospital|zhejiang university|affiliated hospital 1\n" +
                "shenzhen second peoples hospital|shenzhen university|affiliated hospital 1\n" +
                "nanjing medical university|peoples hospital lianyungang 1|affiliated hospital 1\n" +
                "dongguan peoples hospital|southern medical university|affiliated hospital 10\n" +
                "guangdong province hospital chinese medical|guangzhou university chinese medical|affiliated hospital 2\n" +
                "wenzhou medical university|yuying childrens hospital|affiliated hospital 2\n" +
                "shenzhen peoples hospital 3|southern university science & technol|affiliated hospital 2\n" +
                "henan mental hospital|xinxiang medical university|affiliated hospital 2\n" +
                "nantong peoples hospital 1|nantong university|affiliated hospital 2\n" +
                "guangzhou peoples hospital 1|south china university technol|affiliated hospital 2\n" +
                "peoples hospital shenzhen 3|southern university science & technol|affiliated hospital 2\n" +
                "qingdao central hospital|qingdao university|affiliated hospital 2\n" +
                "kunming medical university|yunnan cancer hospital|affiliated hospital 3\n" +
                "peoples hospital zunyi 1|zunyi medical university|affiliated hospital 3\n" +
                "peoples hospital changzhou 1|soochow university|affiliated hospital 3\n" +
                "kunming medical university|tumor hospital yunnan province|affiliated hospital 3\n" +
                "kunming medical university|yunnan tumor hospital|affiliated hospital 3\n" +
                "shaanxi province peoples hospital|xi an jiao tong university|affiliated hospital 3\n" +
                "ruian peoples hospital|wenzhou medical university|affiliated hospital 3\n" +
                "nantong university|peoples hospital yancheng 1|affiliated hospital 4\n" +
                "guangxi medical university|liuzhou workers hospital|affiliated hospital 4\n" +
                "guangzhou medical university|qingyuan peoples hospital|affiliated hospital 6\n" +
                "lishui peoples hospital|wenzhou medical university|affiliated hospital 6\n" +
                "peoples hospital lishui|wenzhou medical university|affiliated hospital 6\n" +
                "kunming medical university|peoples hospital yuxi city|affiliated hospital 6\n" +
                "guangxi medical university|guigang city peoples hospital|affiliated hospital 8\n" +
                "nanjing drum tower hospital|nanjing university|affiliated hospital medical school\n" +
                "nanjing drum tower hospital|nanjing university|affiliated hospital of medical school\n" +
                "huaian peoples hospital 2|xuzhou medical university|affiliated huaian hospital\n" +
                "peoples hospital huaian 2|xuzhou medical university|affiliated huaian hospital\n" +
                "jiangmen central hospital|sun yat sen university|affiliated jiangmen hospital\n" +
                "peoples hospital lianyungang 1|xuzhou medical university|affiliated lianyungang hospital\n" +
                "peoples hospital nantong 6|shanghai university|affiliated nantong hospital\n" +
                "nantong peoples hospital 3|nantong university|affiliated nantong hospital 3\n" +
                "shantou central hospital|sun yat sen university|affiliated shantou hospital\n" +
                "nantong tumor hospital|nantong university|affiliated tumor hospital\n" +
                "southeast university|xuzhou central hospital|affiliated xuzhou hospital\n" +
                "ciudad university|university nacl autonoma mexico|av university 3000\n" +
                "ciudad university|university autonoma aguascalientes|avenue university 940\n" +
                "ciudad university|university autonoma nuevo leon|avenue university s-n\n" +
                "peking union medical college|tsinghua university|beijing\n" +
                "china japan friendship hospital|chinese academy medical science & peking union medical college|beijing\n" +
                "china japan friendship hospital|chinese academy medical science|beijing\n" +
                "peking university|tsinghua university|beijing\n" +
                "capital medical university|china japan friendship hospital|beijing\n" +
                "beijing obstetrics & gynecol hospital|capital medical university|beijing maternal & child health care hospital\n" +
                "central south university|xiangya hospital 2|childrens medical center\n" +
                "guangzhou red cross hospital|jinan university|guangdong\n" +
                "jinan university|zhuhai peoples hospital|guangdong\n" +
                "hospital 3|sun yat sen university|guangdong\n" +
                "jinan university|shenzhen eye hospital|guangdong\n" +
                "affiliated hospital 1|sun yat sen university medical science|guangdong\n" +
                "memorial hospital|sun yat sen university|guangdong\n" +
                "affiliated hospital 1|sun yet sen university|guangdong\n" +
                "brigham & womens hospital|harvard university|harvard medical school\n" +
                "harvard university|massachusetts general hospital|harvard medical school\n" +
                "anhui medical university|peoples hospital hefei 2|hefei hospital\n" +
                "henan province peoples hospital|zhengzhou university|henan university\n" +
                "henan province peoples hospital|zhengzhou university peoples hospital|henan university peoples hospital\n" +
                "shandong first medical university|shandong province qianfoshan hospital|hospital 1\n" +
                "shenzhen peoples hospital 3|southern university science & technol|hospital 2\n" +
                "shanxi bethune hospital|shanxi medical university|hospital 3\n" +
                "shanxi medical university|shanxi province peoples hospital|hospital 5\n" +
                "sichuan university|west china hospital|institute hospital management\n" +
                "affiliated hospital 1|nanchang university|jiangxi medical college\n" +
                "affiliated hospital 2|nanchang university|jiangxi medical college\n" +
                "jinhua municipal central hospital|zhejiang university|jinhua hospital\n" +
                "university hospital leuven|university leuven|ku leuven\n" +
                "tianjin anding hospital|tianjin medical university|mental health center\n" +
                "changhai hospital|naval medical university|military medical university 2\n" +
                "army medical university|southeast hospital|military medical university 3\n" +
                "army medical university|xinqiao hospital|military medical university 3\n" +
                "harvard university|massachusetts general hospital|mit\n" +
                "shanghai childrens medical center|shanghai jiao tong university|nacional childrens medical center\n" +
                "john radcliffe hospital|oxford university hospital nhs fundacion trust|oxford ox3 9du\n" +
                "john radcliffe hospital|oxford university hospital nhs trust|oxford ox3 9du\n" +
                "john radcliffe hospital|oxford university hospital|oxford ox3 9du\n" +
                "quzhou peoples hospital|wenzhou medical university|quzhou affiliated hospital\n" +
                "changhai hospital|naval medical university|second military medical university\n" +
                "shanghai jiao tong university|shanghai peoples hospital 1|shanghai general hospital\n" +
                "fudan university|zhongshan hospital|shanghai medical college\n" +
                "eye & ent hospital|fudan university|shanghai medical college\n" +
                "fudan university|hospital & institute obstetrics & gynecol|shanghai medical college\n" +
                "fudan university|shanghai publ health clinical center|shanghai medical college\n" +
                "cancer hospital|fudan university|shanghai medical college\n" +
                "fudan university|huadong hospital|shanghai medical college\n" +
                "eye ear nose & throat hospital|fudan university|shanghai medical college\n" +
                "fudan university|hospital obstetrics & gynecol|shanghai medical college\n" +
                "shanxi medical university|shanxi province cancer hospital|shanxi hospital\n" +
                "shaoxing peoples hospital|zhejiang university|shaoxing hospital\n" +
                "affiliated shengjing hospital|china medical university|shenyang\n" +
                "china medical university|sheng jing hospital|shenyang\n" +
                "china medical university|shengjing affiliated hospital|shenyang\n" +
                "peoples hospital shunde 1|southern medical university|shunde hospital\n" +
                "peoples hospital shunde foshan 1|southern medical university|shunde hospital\n" +
                "soochow university|suzhou peoples hospital 9|suzhou hospital 9\n" +
                "affiliated hospital 1|soochow university|suzhou medical college\n" +
                "royal north shore hospital|university sydney|sydney medical school\n" +
                "childrens hospital westmead|university sydney|sydney medical school\n" +
                "university sydney|westmead hospital|sydney medical school\n" +
                "fujian medical university|hospital putian 1|teaching hospital\n" +
                "fujian medical university|fujian province cancer hospital|teaching hospital\n" +
                "army medical university|xinqiao hospital|third military medical university\n" +
                "army medical university|daping hospital|third military medical university\n" +
                "affiliated hospital 2|army medical university|third military medical university\n" +
                "affiliated hospital 1|army medical university|third military medical university\n" +
                "huazhong university science & technol|hubei cancer hospital|tongji medical college\n" +
                "huazhong university science & technol|wuhan childrens hospital|tongji medical college\n" +
                "huazhong university science & technol|liyuan hospital|tongji medical college\n" +
                "huazhong university science & technol|wuhan union hospital|tongji medical college\n" +
                "huazhong university science & technol|maternal & child health hospital hubei province|tongji medical college\n" +
                "affiliated tongji hospital|huazhong university science & technol|tongji medical college\n" +
                "huazhong university science & technol|puai hospital|tongji medical college\n" +
                "affiliated union hospital|huazhong university science & technol|tongji medical college\n" +
                "affiliated wuhan mental health center|huazhong university science & technol|tongji medical college\n" +
                "huazhong university of science and technology|xiehe hospital|tongji medical college\n" +
                "hust|tongji hospital|tongji medical college\n" +
                "huazhong university science & technol|wuhan hospital 1|tongji medical college\n" +
                "huazhong university science & technol|wuhan central hospital|tongji medical college\n" +
                "princess margaret hospital|university toronto|university health network\n" +
                "mountain sinai hospital|university toronto|university health network\n" +
                "university medical center utrecht|university utrecht|university weg 100\n" +
                "chongqing health center women & children|chongqing medical university|women & childrens hospital\n" +
                "nanjing matern & child health care hospital|nanjing medical university|womens hospital\n" +
                "nanjing medical university|nanjing women & childrens healthcare hospital|womens hospital\n" +
                "henan childrens hospital|zhengzhou university|zhengzhou childrens hospital\n" +
                "fudan university|shanghai xuhui central hospital|zhongshan xuhui hospital\n" +
                "jinan university|zhuhai peoples hospital|zhuhai hospital\n" +
                "亳州市人民医院|安徽医科大学|安徽理工大学附属亳州医院\n" +
                "河南省人民医院|郑州大学人民医院|河南大学人民医院\n" +
                "河南省儿童医院|郑州大学附属儿童医院|郑州儿童医院";
        String[] split = t.split("\n");
        Set<String[]> removeSet = new HashSet<>();
        for (String s : split) {
            String[] list = s.toLowerCase().trim().split("\\|");
            if (list.length == 3 && !s.startsWith("|") && !s.endsWith("|")) {
                removeSet.add(list);
            }
        }
        return removeSet;
    }

    public static void addKeyWord(String keyWords, double fenQuXiShu, double typeScore, int aid, Map<String, Double> keyWordAndScoreMap, Map<String, Set<Integer>> keyWordAndAidSetMap) {
        if (null != keyWords && keyWords.contains("\"]]")) {
            List<String> strList = JSONObject.parseArray(keyWords, String.class);
            for (String str : strList) {
                if (null != str && str.contains("\"]")) {
                    List<String> list = JSONObject.parseArray(str, String.class);
                    // 1. keyWordSumMap的键是一个list，list内第一位是全写的词，第二位是缩写的词，第三位是类型，如果为"m"，则表示为mesh（即第一位是款目词，第二位是mesh词），如果是全写与缩写则类型为空（即list只有两位），如果list只有一位，表示没有缩写的词，也没有mesh词；
                    String s = list.size() == 3 ? list.get(1).toLowerCase().trim() : "";
                    s = list.size() == 2 ? list.get(0).toLowerCase().trim() : s;
                    s = list.size() == 1 ? list.get(0).toLowerCase().trim() : s;
                    // 以这些介词开头或结尾的关键词，不要，或者单独一个字符开头或结尾的也不要。
                    if (!s.startsWith("the ") && !s.startsWith("and ") && !s.startsWith("in ") && !s.startsWith("of ") && !s.startsWith("at ") && !s.startsWith("to ") && !s.startsWith("on ") && !s.startsWith("this ") && !s.startsWith("from ") && !s.startsWith("-") && !s.startsWith("it is ")
                            && !s.endsWith(" the") && !s.endsWith(" and") && !s.endsWith(" in") && !s.endsWith(" of") && !s.endsWith(" at") && !s.endsWith(" to") && !s.endsWith(" on") && !s.endsWith(" this") && !s.endsWith(" from") && !s.endsWith("-")
                            && !Pattern.compile("^[a-zA-Z0-9] .+?$|.+? [a-zA-Z0-9]$").matcher(s).find() && (s.length() > 8 || s.contains(" "))) {
                        keyWordAndScoreMap.compute(s, (key, value) -> value != null ? (value + typeScore * fenQuXiShu) : typeScore * fenQuXiShu);
                        Utils.putStrSetInt(s, aid, keyWordAndAidSetMap);
                    }
                }
            }
        }
    }

    public static Set<String> getPmDoiSet(String pmArticleTable, int onePageNum, String limit, String autoId, String field) throws SQLException {
        List<Entity> tableNum = Db.use().query("select count(1) as count from " + pmArticleTable);
        Set<String> doiSet = new ConcurrentHashSet<>();
        if (tableNum.get(0).getInt("count") > 0) {
            int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + pmArticleTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + pmArticleTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        String di = Utils.dellEnd(Utils.getStrField(entry, "doi")).toLowerCase();
                        if (di.startsWith("10.") && di.length() > 11) {
                            doiSet.add(di);
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
        return doiSet;
    }

//    public static Map<String, String> getUnitsAndHospNameMap() {
//
//
//    }


}
